{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- make decoder_input by slice target\n",
    "- get next_token by slice output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "decoder_inputs or output\n",
      "[[ 0  1  2  3  4  5  6  7  8  9]\n",
      " [10 11 12 13 14 15 16 17 18 19]\n",
      " [20 21 22 23 24 25 26 27 28 29]\n",
      " [30 31 32 33 34 35 36 37 38 39]]\n",
      "\n",
      "slice decoder_index\n",
      "[[ 5]\n",
      " [15]\n",
      " [25]\n",
      " [35]]\n",
      "\n",
      "slice for target\n",
      "[[ 0  1  2  3  4  5  6  7  8]\n",
      " [10 11 12 13 14 15 16 17 18]\n",
      " [20 21 22 23 24 25 26 27 28]\n",
      " [30 31 32 33 34 35 36 37 38]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "batch_size = 4\n",
    "max_seq_len = 10\n",
    "decoder_index = 5\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "decoder_inputs = tf.reshape(tf.range(0, batch_size*max_seq_len), [batch_size, max_seq_len])\n",
    "\n",
    "print(\"decoder_inputs or output\")\n",
    "print(decoder_inputs.eval())\n",
    "print()\n",
    "\n",
    "next_token = tf.slice(decoder_inputs, [0, decoder_index], [batch_size, 1])\n",
    "\n",
    "print(\"slice decoder_index\")\n",
    "print(next_token.eval())\n",
    "print()\n",
    "\n",
    "target_slice_last_1 = tf.slice(decoder_inputs, [0, 0], [batch_size, max_seq_len-1])\n",
    "\n",
    "print(\"slice for target\")\n",
    "print(target_slice_last_1.eval())\n",
    "print()\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- mode : Evaluattion or Predict\n",
    "- predict output with loop. [encoder_outputs, decoder_inputs (filled next token)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "decoder_inputs.\n",
      "[[ 0  1  2  3  4  0  0  0  0  0]\n",
      " [ 5  6  7  8  9  0  0  0  0  0]\n",
      " [10 11 12 13 14  0  0  0  0  0]\n",
      " [15 16 17 18 19  0  0  0  0  0]]\n",
      "\n",
      "next_token\n",
      "[[ 0  0  0  0  0 21  0  0  0  0]\n",
      " [ 0  0  0  0  0 22  0  0  0  0]\n",
      " [ 0  0  0  0  0 23  0  0  0  0]\n",
      " [ 0  0  0  0  0 24  0  0  0  0]]\n",
      "\n",
      "next decoder_inputs.\n",
      "[[ 0  1  2  3  4 21  0  0  0  0]\n",
      " [ 5  6  7  8  9 22  0  0  0  0]\n",
      " [10 11 12 13 14 23  0  0  0  0]\n",
      " [15 16 17 18 19 24  0  0  0  0]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "batch_size = 4\n",
    "max_seq_len = 10\n",
    "decoder_index = 5\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "decoder_inputs = tf.reshape(tf.range(0, 20), [batch_size, 5])\n",
    "zero_pads = tf.zeros([batch_size, 5], dtype=tf.int32)\n",
    "decoder_inputs = tf.concat((decoder_inputs, zero_pads), axis=1)\n",
    "\n",
    "print(\"decoder_inputs.\")\n",
    "print(decoder_inputs.eval())\n",
    "print()\n",
    "\n",
    "# next_token [21, 22, 23, 24] \n",
    "# -> make tensor shape [batch_size, max_seq_len], have value decoder_index\n",
    "next_token = tf.reshape(tf.constant([21, 22, 23, 24]), [batch_size, 1])\n",
    "left_zero_pads = tf.zeros([batch_size, decoder_index], dtype=tf.int32)\n",
    "right_zero_pads = tf.zeros([batch_size, (max_seq_len-decoder_index-1)], dtype=tf.int32)\n",
    "\n",
    "next_token = tf.concat((left_zero_pads, next_token, right_zero_pads), axis=1)\n",
    "\n",
    "print(\"next_token\")\n",
    "print(next_token.eval())\n",
    "print()\n",
    "\n",
    "decoder_inputs = decoder_inputs + next_token\n",
    "# We can just use 'c.eval()' without passing 'sess'\n",
    "\n",
    "print(\"next decoder_inputs.\")\n",
    "print(decoder_inputs.eval())\n",
    "print()\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "decoder_inputs.\n",
      "[[ 0  1  2  3  4  0  0  0  0  0]\n",
      " [ 5  6  7  8  9  0  0  0  0  0]\n",
      " [10 11 12 13 14  0  0  0  0  0]\n",
      " [15 16 17 18 19  0  0  0  0  0]]\n",
      "\n",
      "next_token\n",
      "[[ 0  0  0  0  0  0  0  0  0 21]\n",
      " [ 0  0  0  0  0  0  0  0  0 22]\n",
      " [ 0  0  0  0  0  0  0  0  0 23]\n",
      " [ 0  0  0  0  0  0  0  0  0 24]]\n",
      "\n",
      "next decoder_inputs.\n",
      "[[ 0  1  2  3  4  0  0  0  0 21]\n",
      " [ 5  6  7  8  9  0  0  0  0 22]\n",
      " [10 11 12 13 14  0  0  0  0 23]\n",
      " [15 16 17 18 19  0  0  0  0 24]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "batch_size = 4\n",
    "max_seq_len = 10\n",
    "decoder_index = 9\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "decoder_inputs = tf.reshape(tf.range(0, 20), [batch_size, 5])\n",
    "zero_pads = tf.zeros([batch_size, 5], dtype=tf.int32)\n",
    "decoder_inputs = tf.concat((decoder_inputs, zero_pads), axis=1)\n",
    "\n",
    "print(\"decoder_inputs.\")\n",
    "print(decoder_inputs.eval())\n",
    "print()\n",
    "\n",
    "# next_token [21, 22, 23, 24] \n",
    "# -> make tensor shape [batch_size, max_seq_len], have value decoder_index\n",
    "next_token = tf.reshape(tf.constant([21, 22, 23, 24]), [batch_size, 1])\n",
    "left_zero_pads = tf.zeros([batch_size, decoder_index], dtype=tf.int32)\n",
    "right_zero_pads = tf.zeros([batch_size, (max_seq_len-decoder_index-1)], dtype=tf.int32)\n",
    "\n",
    "next_token = tf.concat((left_zero_pads, next_token, right_zero_pads), axis=1)\n",
    "\n",
    "print(\"next_token\")\n",
    "print(next_token.eval())\n",
    "print()\n",
    "\n",
    "decoder_inputs = decoder_inputs + next_token\n",
    "# We can just use 'c.eval()' without passing 'sess'\n",
    "\n",
    "print(\"next decoder_inputs.\")\n",
    "print(decoder_inputs.eval())\n",
    "print()\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 (NLP)",
   "language": "python",
   "name": "nlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
